package com.chance.cc.crawler.development.scripts.baidu.yidianzixun;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.http.TruncatedChunkException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.SocketTimeoutException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class YiDianCrawlerScript extends CrawlerCommonScript {

    private static Logger logger = LoggerFactory.getLogger(YiDianCrawlerScript.class);

    public static final String beginUrl = "https://www.baidu.com/s\\?ie=utf-8&f=8&rsv_bp=1&rsv_idx=2&tn=baiduhome_pg&wd=nike&ct=2097152&si=yidianzixun.com&rsv_spt=1&oq=%25E4%25BD%25A0%25E5%258F%25AF&rsv_pq=ee20d1a400086822&rsv_t=\\S*&rqlang=cn&rsv_enter=1&rsv_dl=tb&rsv_btype=t&inputT=2580&rsv_sug3=7&rsv_sug1=7&rsv_sug7=100&rsv_jmp=slow";

    public static final String firstListUrl = "https://www.baidu.com/s\\?wd=\\S*&pn=\\d*&oq=\\S*&ct=\\d*&tn=baiduhome_pg&ie=utf-8&si=yidianzixun.com&rsv_idx=2&gpc=\\S*&tfflag=\\S*";

    public static final String ListUrl = "https://www.baidu.com/s\\?wd=\\S*&pn=\\d*&oq=\\S*&ct=\\d*&tn=baiduhome_pg&ie=utf-8&si=yidianzixun.com&rsv_idx=2&rsv_pq=\\S*&rsv_t=\\S*&gpc=\\S*&tfflag=\\S*";

    public static final String articleUrl = "http://www.yidianzixun.com/article/\\S*";

    public static final String anotherArticleUrl = "http://www.yidianzixun.com/article/\\S*\\?s=\\S*&appid=\\S*";

    public static final String otherArticleUrl = "http://www.yidianzixun.com/article/\\S*/amp";

    public static final String lastOneArticleUrl = "http://www.yidianzixun.com/article/\\S*appid=\\S*";


    public static final String changeUrl = "http://www.baidu.com/link\\?url=\\S*";

    private static final String DOMAIN = "baidu";

    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    public static final String site = "YDZX";


    @Override
    public String domain() {
        return "baidu";
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();
        String crawlerSite = categoryTag.getKVTagStrVal("site");
        return site.equalsIgnoreCase(crawlerSite);
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(beginUrl);
        addUrlRegular(ListUrl);
        addUrlRegular(articleUrl);
        addUrlRegular(changeUrl);
        addUrlRegular(firstListUrl);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<>();

        if (doHttpPageCheck(crawlerRequestRecord, httpPage)) {
            requestAgainCrawlerRecord(parsedLinks, crawlerRequestRecord);
            crawlerRequestRecord.setNeedWashPage(false);
            return parsedLinks;
        }
        String url = httpPage.getRequest().getUrl();


        if (url.matches(beginUrl)) {
            return parseBeginLinks(crawlerRequestRecord, httpPage);
        }
        if (url.matches(firstListUrl) || url.matches(ListUrl)) {
            return parseListLinks(crawlerRequestRecord, httpPage);
        }

        return parsedLinks;
    }


    private boolean doHttpPageCheck(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {

        String lastRequestUrl = crawlerRequestRecord.getHttpRequest().getUrl();
        int statusCode = httpPage.getStatusCode();

        if (statusCode != 200) {
            String s1 = httpPage.getHtml().xpath("//div[@class=\"left-wrapper content-empty\"]/p").get();
            if (StringUtils.isEmpty(s1)) {
                s1 = httpPage.getHtml().xpath("//div[@class=\"hint\"]/text()").get();
            }
            if (StringUtils.isNotEmpty(s1)) {
                return false;
            }
            logger.error("download page {} error, status code is {}", lastRequestUrl, statusCode);
            return true;
        }

        boolean downloadSuccess = httpPage.isDownloadSuccess();
        if (!downloadSuccess) {
            logger.error("download page {} error", lastRequestUrl);
            return true;
        }

        String s = httpPage.getHtml().get();
        // 有时候下载下一页的列表页，会解析不了xpath， 出现百度安全验证的词，就需要重新下载一遍
        if (s.contains("百度安全验证")) {
            return true;
        }
        return false;
    }

    private List<CrawlerRequestRecord> parseListLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<>();
        List<Selectable> nodes = httpPage.getHtml().xpath("//div[@class=\"result c-container new-pmd\"]").nodes();

        for (Selectable node : nodes) {
            String time = node.xpath(".//div[@class=\"c-abstract\"]/span").get();
            String itemUrl = node.xpath("./h3/a/@href").get();
            long timeCount = getPubTime(time);

            CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .httpUrl(itemUrl)
                    .recordKey(itemUrl)
                    .releaseTime(timeCount)
                    .resultLabelTag(article)
                    .copyBizTags()
                    .build();
            itemRecord.setNeedWashPage(true);
            itemRecord.tagsCreator().bizTags().addCustomKV("time", timeCount);
            parsedLinks.add(itemRecord);

        }


        String recordUrl = httpPage.getRequest().getUrl();
        String pn = recordUrl.substring(recordUrl.indexOf("&pn=") + 4, recordUrl.indexOf("&oq"));
        int count = (Integer.parseInt(pn) / 10) + 1;

        List<String> all = httpPage.getHtml().xpath("//span[@class=\"page-item_M4MDr pc\"]/text()").all();


        if (all.size() != 0) {
            String pageNumber = all.get(all.size() - 1);
            if (Integer.parseInt(pageNumber) > count) {
                String url = httpPage.getHtml().xpath("//div[@class=\"page-inner_2jZi2\"]/a[2]/@href").get();


                pn = (Integer.parseInt(pn) + 10) + "";
                String rsv_pq = url.substring(url.indexOf("rsv_pq=") + 7, url.indexOf("rsv_t") - 5);
                String rsv_t = url.substring(url.indexOf("rsv_t=") + 6, url.indexOf("gpc") - 5);


                String listUrl = httpPage.getRequest().getUrl();
                String oq = listUrl.substring(listUrl.indexOf("oq") + 3, listUrl.indexOf("ct") - 1);
                String ct = listUrl.substring(listUrl.indexOf("ct") + 3, listUrl.indexOf("tn") - 1);
                String time = listUrl.substring(listUrl.indexOf("stf%3D") + 6, listUrl.indexOf("%7Cstftype%3D1"));
                String startTime = time.split("%2C")[0];
                String endTime = time.split("%2C")[1];

                String nextUrl = "https://www.baidu.com/s?wd=" + oq + "&pn=" + pn + "&oq=" + oq + "&ct=" + ct + "&tn=baiduhome_pg&ie=utf-8&si=yidianzixun.com&rsv_idx=2&rsv_pq=" + rsv_pq + "&rsv_t=" + rsv_t + "&gpc=stf%3D" + startTime + "%2C" + endTime + "%7Cstftype%3D1&tfflag=1&topic_pn=";
                // String nextUrl = "https://www.baidu.com/s?wd="+oq+"&pn=160&oq="+oq+"&ct="+ct+"&tn=baiduhome_pg&ie=utf-8&si=yidianzixun.com&rsv_idx=2&rsv_pq="+rsv_pq+"&rsv_t="+rsv_t+"&gpc=stf%3D"+startTime+"%2C"+endTime+"%7Cstftype%3D1&tfflag=1&topic_pn=" ;

                CrawlerRequestRecord listRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .httpUrl(nextUrl)
                        .recordKey(nextUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .copyResultTags()
                        .build();
                parsedLinks.add(listRecord);

            }
        }
        return parsedLinks;
    }

    public long getPubTime(String pubTime) {
        long timeNumber = 0;
        if (pubTime.contains("小时前")) {
            String time = pubTime.split("小时")[0];
            long l = System.currentTimeMillis();
            int i = Integer.parseInt(time) * 60 * 60 * 1000;
            timeNumber = l - i;
        } else if (pubTime.contains("天前")) {
            String time = pubTime.split("天前")[0];
            long l = System.currentTimeMillis();
            int i = Integer.parseInt(time) * 24 * 60 * 60 * 1000;
            timeNumber = l - i;
        } else if (pubTime.contains("分钟前")) {
            String time = pubTime.split("分钟")[0];
            long l = System.currentTimeMillis();
            int i = Integer.parseInt(time) * 60 * 1000;
            timeNumber = l - i;
        } else if (pubTime.contains("年") && pubTime.contains("月")) {
            try {
                pubTime = pubTime.substring(0, pubTime.indexOf("日") + 1);
                timeNumber = DateUtils.parseDate(pubTime, "yyyy年MM月dd日").getTime();
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }

        return timeNumber;
    }

    private List<CrawlerRequestRecord> parseBeginLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {

        List<CrawlerRequestRecord> parsedLinks = new ArrayList<>();
        long endTime = System.currentTimeMillis() / 1000;
        long startTime = endTime - 7 * 24 * 60 * 60;


        String nextPageUrl = "https://www.baidu.com/s?wd=nike&pn=0&oq=nike&ct=2097152&tn=baiduhome_pg&ie=utf-8&si=yidianzixun.com&rsv_idx=2&gpc=stf%3D" + startTime + "%2C" + endTime + "%7Cstftype%3D1&tfflag=1&rsv_jmp=slow";
        CrawlerRequestRecord nextRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .httpUrl(nextPageUrl)
                .recordKey(nextPageUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .copyResultTags()
                .needWashed(false)
                .build();
        parsedLinks.add(nextRecord);
        return parsedLinks;

    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {

        if (crawlerRequestRecord.tagsCreator().resultTags().hasDataType(article)) {
            return washArticle(crawlerRequestRecord, httpPage);
        }

        return null;
    }

    private List<CrawlerData> washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String time = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("time");

        List<CrawlerData> crawlerDataList = new ArrayList<>();
        String s1 = httpPage.getHtml().xpath("//div[@class=\"left-wrapper content-empty\"]/p").get();
        if (StringUtils.isEmpty(s1)) {
            s1 = httpPage.getHtml().xpath("//div[@class=\"hint\"]/text()").get();
        }
        if (StringUtils.isEmpty(s1)) {
            String text = httpPage.getHtml().get();
            String title = "";
            title = httpPage.getHtml().xpath("//header[@id=\"header\"]/h1/text()").get();
            if (StringUtils.isEmpty(title)) {
                title = httpPage.getHtml().xpath("//h2[@id=\"pc-article-title\"]/text()").get();
            }
            if (StringUtils.isEmpty(title)) {
                title = httpPage.getHtml().xpath("//div[@class=\"left-wrapper\"]/h2/text()").get();
            }
            if (StringUtils.isEmpty(title)) {
                title = httpPage.getHtml().xpath("//header[@id=\"header\"]/h3/text()").get();
            }
            if (StringUtils.isEmpty(title)) {
                title = httpPage.getHtml().xpath("//h1/text()").get();
            }
            if(StringUtils.isEmpty(title)){
                title = new Html(text).xpath("//title").get();
            }


            String author = "";
            author = httpPage.getHtml().xpath("//header[@id=\"header\"]/div[@class=\"meta\"]/span[@class=\"source\"]/text()").get();
            if (StringUtils.isEmpty(author)) {
                author = httpPage.getHtml().xpath("//div[@class=\"meta\"]/span[@class=\"doc-source\"]/text()").get();
            }
            if (StringUtils.isEmpty(author)) {
                author = httpPage.getHtml().xpath("//div[@class=\"meta\"]/span[1]/text()").get();
                if (StringUtils.isNotEmpty(author)) {
                    if (author.contains("-")) {
                        author = httpPage.getHtml().xpath("//div[@class=\"meta\"]/span[2]/text()").get();
                    }
                }
            }

            if (StringUtils.isEmpty(author)) {
                author = httpPage.getHtml().xpath("//span[@class=\"source yidian-wemedia-source\"]").get();
            }


            //http://www.baidu.com/link?url=s8QIETrxvnmBR3oX5VE1E3bN6Ev1ub_dcFWN8wbax1mXr4g_p1dAjOSGb9zpCl2NCqEjwYJf_y_N00ds3yFuNq
            String articleUrl = crawlerRequestRecord.getHttpRequest().getUrl();

            String url = httpPage.getHtml().xpath("//link[@rel=\"canonical\"]/@href").get();
            if (StringUtils.isEmpty(url)) {
                httpPage.getHtml().xpath("/html/head/link[1]");
            }

            //http://www.yidianzixun.com/article/V_0ZAkY2sl?appid=s3rd_op398
            String articleId = "";
            if (StringUtils.isNotEmpty(url)) {
                if (url.matches(anotherArticleUrl)) {
                    articleId = url.substring(url.indexOf("article/") + 8, url.indexOf("?"));
                } else if (url.matches(otherArticleUrl)) {
                    articleId = url.substring(url.indexOf("article/") + 8, url.lastIndexOf("/"));
                } else if (url.matches(lastOneArticleUrl)) {
                    articleId = url.substring(url.indexOf("article/") + 8, url.lastIndexOf("?"));
                } else {
                    articleId = url.substring(url.indexOf("article/") + 8);
                }
            }

            if (StringUtils.isEmpty(url)) {

                if (StringUtils.isNotEmpty(text) && !text.contains("__docId__")) {
                    if (text.contains("yidian.docinfo.doc")) {
                        String s = text.substring(text.indexOf("yidian.docinfo") + 17, text.indexOf("yidian.docinfo.doc"));
                        s = s.substring(0, s.lastIndexOf(";"));
                        JSONObject jsonObject = JSONObject.parseObject(s);
                        JSONObject jsonObject1 = JSONObject.parseObject(jsonObject.getString("query"));
                        articleId = jsonObject1.getString("id");
                        if (StringUtils.isEmpty(articleId)) {
                            articleId = jsonObject1.getString("docid");
                        }
                    } else if (text.contains("window.yidian.docinfo") && text.contains("is_debug")) {
                        String s = text.substring(text.indexOf("window.yidian.docinfo") + 24, text.indexOf("is_debug") - 2);
                        s = s + "}";
                        JSONObject jsonObject = JSONObject.parseObject(s);
                        JSONObject jsonObject1 = JSONObject.parseObject(jsonObject.getString("query"));
                        articleId = jsonObject1.getString("id");
                    }
                } else {
                    articleId = text.substring(text.indexOf("__docId__") + 12, text.indexOf("querystring") - 23);
                }


                if (StringUtils.isEmpty(articleId)) {
                    String substring = text.substring(text.indexOf("var url = ") + 10, text.indexOf("if(s == 'vivobrowser')"));
                    articleId = substring.substring(substring.lastIndexOf("/")+1);

                }

            }


            List<String> all = httpPage.getHtml().xpath("//div[@id=\"content\"]//text()").all();
            if (all.size() == 0) {
                all = httpPage.getHtml().xpath("//div[@class=\"content-bd\"]//text()").all();
            }
            if (all.size() == 0) {
                all = httpPage.getHtml().xpath("//div[@id=\"content\"]/text()").all();
            }
            if (all.size() == 0) {
                all = httpPage.getHtml().xpath("//div[@id=\"imedia-article\"]/p").all();
            }
            if (all.size() == 0) {
                all = new Html(text).xpath("//div[@class=\"content\"]/p/text()").all();
            }

            StringBuffer contentText = new StringBuffer();
            for (String s : all) {
                contentText.append(s);
            }


            List<String> pictureList = httpPage.getHtml().xpath("//article[@id=\"article\"]//amp-img/@src").all();
            if (pictureList.size() == 0) {
                pictureList = httpPage.getHtml().xpath("//div[@class=\"content-bd\"]//img/@src").all();
            }
            if (pictureList.size() == 0) {
                pictureList = httpPage.getHtml().xpath("//div[@id=\"content\"]//img[@decoding=\"async\"]/@src").all();
            }
            if (pictureList.size() == 0) {
                pictureList = new Html(text).xpath("//div[@class=\"content\"]//img/@src").all();
            }
            StringBuffer pictures = new StringBuffer();
            for (String picture : pictureList) {
                if (picture.contains("http:")) {
                    pictures.append(picture).append("/0X1");
                } else {
                    pictures.append("https:").append(picture).append("/0X1");
                }
            }

            CrawlerData articleData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleId))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .url(articleUrl)
                    .releaseTime(Long.parseLong(time))
                    .addContentKV(AICCommonField.Field_Content, contentText.toString().trim())
                    .addContentKV(AICCommonField.Field_Title, title)
                    .addContentKV(AICCommonField.Field_Author, author)
                    .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
                    .build();


            crawlerDataList.add(articleData);
        }

        return crawlerDataList;
    }


    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 30) {
                logger.error("autohome download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
